from __future__ import division
from operator import itemgetter, attrgetter
from struct import *
import copy
import math
import os
import random
import sys
import time
from sets import Set
from random import randint
import re
import numpy as np
from os import walk
from subprocess import call
import os

def function1():
    # from this, we can sth done
    print "Begins..."
    # ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/point2MQueries_postingHits_20141106_final.binary"
    # ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/50MQueries/binaryFiles/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary"
    # do the 1%,2%,3%,4%,5%,6% and 7% for the posting hit
    ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_lastMile_20141109_final.binary_theLastPointPercentage"
    # ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_2%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_3%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_4%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_5%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_6%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_7%"
    ofh = open(ofn,"wb")
    
    # point2M fake Queries - completed
    # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/point2MQueries/binaryFiles/resultfile0_point2MQueries_postingHits_20141106_final"
    # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/10MQueries/binaryFiles/resultfile0_10MQueries_postingHits_20141106_final"
    # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/50MQueries/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final"
    # do the 1%,2%,3%,4%,5%,6% and 7% for the posting hit
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_1%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_2%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_3%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_4%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_5%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_6%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_7%"
    ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/50MQueries_TOP1000/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final.binary0"
    ifh0 = open(ifn0,"rb")
    
    fileList = []
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_0M_1M"
    ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_1M_2M"
    ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_2M_3M"
    ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_3M_4M"
    ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_4M_5M"
    ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_5M_6M"
    ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_6M_7M"
    ifn8 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_7M_8M"
    ifn9 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_8M_9M"
    ifn10 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_9M_10M"
    ifn11 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_10M_11M"
    ifn12 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_11M_12M"
    ifn13 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_12M_13M"
    ifn14 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_13M_14M"
    ifn15 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_14M_15M"
    ifn16 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_15M_16M"
    ifn17 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_16M_17M"
    ifn18 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_17M_18M"
    ifn19 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_18M_19M"
    ifn20 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_19M_20M"
    ifn21 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_20M_21M"
    ifn22 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_21M_22M"
    ifn23 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_22M_23M"
    ifn24 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_23M_24M"
    ifn25 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_24M_25M"
    ifn26 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_25M_END"
    
    
    fileList.append(ifn1)
    fileList.append(ifn2)
    fileList.append(ifn3)
    fileList.append(ifn4)
    fileList.append(ifn5)
    fileList.append(ifn6)
    fileList.append(ifn7)
    fileList.append(ifn8)
    fileList.append(ifn9)
    fileList.append(ifn10)
    fileList.append(ifn11)
    fileList.append(ifn12)
    fileList.append(ifn13)
    fileList.append(ifn14)
    fileList.append(ifn15)
    fileList.append(ifn16)
    fileList.append(ifn17)
    fileList.append(ifn18)
    fileList.append(ifn19)
    fileList.append(ifn20)
    fileList.append(ifn21)
    fileList.append(ifn22)
    fileList.append(ifn23)
    fileList.append(ifn24)
    fileList.append(ifn25)
    fileList.append(ifn26)
    
    fileCounter = 0
    notEmptyFlag = True
    
    # let's have the BM25 impact score ready through the scanning of the document
    inputFileHandler0 = open(fileList[fileCounter],"rb")
    statinfo = os.stat(fileList[fileCounter])
    fileSize = statinfo.st_size
    print "currentInputFileName: ",fileList[fileCounter]
    print "file size:",fileSize
    numOfBytesRead = 0
    
    currentProcessingDocID = -1
    currentProcessingDocIDPostingTupleList = []
    byteString = ifh0.read(4 + 4 + 4)
    
    while cl:
        
        (currentID,currentTermID,_) = unpack( "2If", byteString)
        if currentID != currentProcessingDocID:
                    
            # init
            currentProcessingDocID = -1
            currentProcessingDocIDPostingTupleList = []
            
            # assignment
            currentProcessingDocID = currentID
            currentProcessingDocIDPostingTupleList.append(currentTermID)        
            
            byteString = ifh0.read(4 + 4 + 4)
            if cl.strip() != "":
                (currentID,currentTermID,_) = unpack( "2If", byteString)
                while currentID == currentProcessingDocID and notEmptyFlag:
                    currentProcessingDocIDPostingTupleList.append(currentTermID)
                    (currentID,currentTermID,_) = unpack( "2If", byteString)
                    if cl.strip() != "":
                        pass
                    else:
                        # in total
                        notEmptyFlag = False
                        print currentProcessingDocID,currentProcessingDocIDPostingTupleList                    
                        # currentProcessingDocID = -1
                        # currentProcessingDocIDPostingTupleList = []
                                            
            else:
                # in total
                print currentProcessingDocID,currentProcessingDocIDPostingTupleList            
                # currentProcessingDocID = -1
                # currentProcessingDocIDPostingTupleList = []
                        
            
            # in total
            if currentProcessingDocID != -1:
                print currentProcessingDocID,currentProcessingDocIDPostingTupleList
        
            # let's do another source file
            if numOfBytesRead == fileSize:
                inputFileHandler0.close()
                fileCounter += 1
                inputFileHandler0 = open(fileList[fileCounter],"rb")
                statinfo = os.stat(fileList[fileCounter])
                fileSize = statinfo.st_size
                print "currentInputFileName: ",fileList[fileCounter]
                print "file size:",fileSize
                numOfBytesRead = 0
                
                
            while numOfBytesRead < fileSize:
                byteString = inputFileHandler0.read(4 + 4)
                (docID,numOfPostings) = unpack( "2I", byteString)
                print "docID:",docID,numOfPostings
                
                if docID == currentProcessingDocID:
                    verificationFactor = 0
                    for i in range(0,numOfPostings):
                        byteString = inputFileHandler0.read(4 + 4 + 4 + 4 + 4)
                        (termID,static,dynamic,combined,score1) = unpack( "1I4f", byteString)
                        if termID in currentProcessingDocIDPostingTupleList:
                            print "----->",i,termID,static,dynamic,combined,score1
                            ofh.write(pack("2If",docID,termID,score1))
                            verificationFactor += 1
                    if verificationFactor == len(currentProcessingDocIDPostingTupleList):
                        print "good to go"
                    else:
                        print "verificationFactor:",verificationFactor
                        print "len(currentProcessingDocIDPostingTupleList):",len(currentProcessingDocIDPostingTupleList)
                        exit(1)
                    numOfBytesRead += 8 + numOfPostings * 4 * 5
                    inputFileHandler0.seek(numOfBytesRead)                
                    break
                
                numOfBytesRead += 8 + numOfPostings * 4 * 5
                inputFileHandler0.seek(numOfBytesRead)
    ifh0.close()
    ofh.close()
    print "Overall:"
    print "ofn:",ofn
    print "ifn0:",ifn0
    print "End."

def function2():
    # from this, we can sth done
    print "Begins..."
    # ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/point2MQueries_postingHits_20141106_final.binary"
    # ofn = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/50MQueries/binaryFiles/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary"
    # do the 1%,2%,3%,4%,5%,6% and 7% for the posting hit
    ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141110"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141109"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_2%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_3%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_4%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_5%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_6%"
    #ofn = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_lastMile_20141108_final.binary_7%"
    ofh = open(ofn,"wb")
    
    # point2M fake Queries - completed
    # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/point2MQueries/binaryFiles/resultfile0_point2MQueries_postingHits_20141106_final"
    # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/10MQueries/binaryFiles/resultfile0_10MQueries_postingHits_20141106_final"
    # ifn0 = "/home/vgc/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/postingHits_20141031/50MQueries/binaryFiles/resultfile0_50MQueries_postingHits_20141106_final"
    # do the 1%,2%,3%,4%,5%,6% and 7% for the posting hit
    ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141109_final"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141109_final"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_1%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_2%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_3%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_4%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_5%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_6%"
    #ifn0 = "/san_data/research/wei/workspace/NYU_IRTK/polyIRToolkit_Wei/resultfile0_50MQueries_postingHits_20141106_final_sortedByDocHit_20141108_7%"
    
    
    ifh0 = open(ifn0,"r")
    
    fileList = []
    ifn1 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_0M_1M"
    ifn2 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_1M_2M"
    ifn3 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_2M_3M"
    ifn4 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_3M_4M"
    ifn5 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_4M_5M"
    ifn6 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_5M_6M"
    ifn7 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_6M_7M"
    ifn8 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_7M_8M"
    ifn9 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_8M_9M"
    ifn10 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_9M_10M"
    ifn11 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_10M_11M"
    ifn12 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_11M_12M"
    ifn13 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_12M_13M"
    ifn14 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_13M_14M"
    ifn15 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_14M_15M"
    ifn16 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_15M_16M"
    ifn17 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_16M_17M"
    ifn18 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_17M_18M"
    ifn19 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_18M_19M"
    ifn20 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_19M_20M"
    ifn21 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_20M_21M"
    ifn22 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_21M_22M"
    ifn23 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_22M_23M"
    ifn24 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_23M_24M"
    ifn25 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_24M_25M"
    ifn26 = "/home/vgc/wei/workspace/NYU_IRTK/data/unigramFromWei/documentPostingArrays/gov2/essentialityFactorAdded_20140929/GOV2_documentPostingArray_25M_END"
    
    
    fileList.append(ifn1)
    fileList.append(ifn2)
    fileList.append(ifn3)
    fileList.append(ifn4)
    fileList.append(ifn5)
    fileList.append(ifn6)
    fileList.append(ifn7)
    fileList.append(ifn8)
    fileList.append(ifn9)
    fileList.append(ifn10)
    fileList.append(ifn11)
    fileList.append(ifn12)
    fileList.append(ifn13)
    fileList.append(ifn14)
    fileList.append(ifn15)
    fileList.append(ifn16)
    fileList.append(ifn17)
    fileList.append(ifn18)
    fileList.append(ifn19)
    fileList.append(ifn20)
    fileList.append(ifn21)
    fileList.append(ifn22)
    fileList.append(ifn23)
    fileList.append(ifn24)
    fileList.append(ifn25)
    fileList.append(ifn26)
    
    fileCounter = 0
    notEmptyFlag = True
    
    # let's have the BM25 impact score ready through the scanning of the document
    inputFileHandler0 = open(fileList[fileCounter],"rb")
    statinfo = os.stat(fileList[fileCounter])
    fileSize = statinfo.st_size
    print "currentInputFileName: ",fileList[fileCounter]
    print "file size:",fileSize
    numOfBytesRead = 0
            
    currentProcessingDocID = -1
    currentProcessingDocIDPostingTupleList = {}
    cl = ifh0.readline()
    while cl:
        cle = cl.strip().split(" ")
        currentID = int(cle[0])
        currentTermID = int(cle[1])
        if currentID != currentProcessingDocID:
                    
            # init
            currentProcessingDocID = -1
            currentProcessingDocIDPostingTupleList = {}
            
            # assignment
            currentProcessingDocID = currentID
            currentProcessingDocIDPostingTupleList[currentTermID] = 1        
            
            cl = ifh0.readline()
            if cl.strip() != "":
                cle = cl.strip().split(" ")
                currentID = int(cle[0])
                currentTermID = int(cle[1])
                      
                while currentID == currentProcessingDocID and notEmptyFlag:
                    currentProcessingDocIDPostingTupleList[currentTermID] = 1
                    cl = ifh0.readline()
                    if cl.strip() != "":
                        cle = cl.strip().split(" ")
                        currentID = int(cle[0])
                        currentTermID = int(cle[1])
                    else:
                        # in total
                        notEmptyFlag = False
                        # print currentProcessingDocID,currentProcessingDocIDPostingTupleList                    
                        # currentProcessingDocID = -1
                        # currentProcessingDocIDPostingTupleList = []
                                            
            else:
                # in total
                # print currentProcessingDocID,currentProcessingDocIDPostingTupleList            
                # currentProcessingDocID = -1
                # currentProcessingDocIDPostingTupleList = []
                pass      
            
            # in total
            if currentProcessingDocID != -1:
                # print currentProcessingDocID,currentProcessingDocIDPostingTupleList
                pass
            
            # let's do another source file
            if numOfBytesRead == fileSize:
                inputFileHandler0.close()
                fileCounter += 1
                inputFileHandler0 = open(fileList[fileCounter],"rb")
                statinfo = os.stat(fileList[fileCounter])
                fileSize = statinfo.st_size
                print "currentInputFileName: ",fileList[fileCounter]
                print "file size:",fileSize
                numOfBytesRead = 0
                
                
            while numOfBytesRead < fileSize:
                byteString = inputFileHandler0.read(4 + 4)
                (docID,numOfPostings) = unpack( "2I", byteString)
                print "docID:",docID,numOfPostings
                
                if docID == currentProcessingDocID:
                    verificationFactor = 0
                    for i in range(0,numOfPostings):
                        byteString = inputFileHandler0.read(4 + 4 + 4 + 4 + 4)
                        (termID,static,dynamic,combined,score1) = unpack( "1I4f", byteString)
                        if termID in currentProcessingDocIDPostingTupleList:
                            # print "----->",i,termID,static,dynamic,combined,score1
                            ofh.write(pack("2If",docID,termID,score1))
                            verificationFactor += 1
                    if verificationFactor == len(currentProcessingDocIDPostingTupleList):
                        print "good to go"
                        # exit(1)
                    else:
                        print "verificationFactor:",verificationFactor
                        print "len(currentProcessingDocIDPostingTupleList):",len(currentProcessingDocIDPostingTupleList)
                        # exit(1)
                    numOfBytesRead += 8 + numOfPostings * 4 * 5
                    inputFileHandler0.seek(numOfBytesRead)                
                    break
                
                numOfBytesRead += 8 + numOfPostings * 4 * 5
                inputFileHandler0.seek(numOfBytesRead)
    ifh0.close()
    ofh.close()
    print "Overall:"
    print "ofn:",ofn
    print "ifn0:",ifn0
    print "End."

print "Begins:"
# function1()
function2()
print "Ends."



